{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 文本分类实例"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step1 导入相关包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\numpy\\_distributor_init.py:30: UserWarning: loaded more than 1 DLL from .libs:\n",
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\numpy\\.libs\\libopenblas.FB5AE2TYXYH2IJRDKGDGQ3XBKLKTF43H.gfortran-win_amd64.dll\n",
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\numpy\\.libs\\libopenblas64__v0.3.21-gcc_10_3_0.dll\n",
      "  warnings.warn(\"loaded more than 1 DLL from .libs:\"\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments, BertTokenizer, BertForSequenceClassification\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pretrain_model_dir = \"/data/models/huggingface/rbt3\"\n",
    "save_dir = '/data/logs/data_parallel_train'"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step2 加载数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['label', 'review'],\n",
       "    num_rows: 7765\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = load_dataset(\"csv\", data_files=\"./ChnSentiCorp_htl_all.csv\", split=\"train\")\n",
    "dataset = dataset.filter(lambda x: x[\"review\"] is not None)\n",
    "dataset"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step3 划分数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['label', 'review'],\n",
       "        num_rows: 6988\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['label', 'review'],\n",
       "        num_rows: 777\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets = dataset.train_test_split(test_size=0.1)\n",
    "datasets"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step4 数据集预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15cb6488f0a04d5a87b35eafc5303114",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/6988 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a37bc0b96d564a94921b7c9b6f9264b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/777 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 6988\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 777\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(pretrain_model_dir)\n",
    "\n",
    "def process_function(examples):\n",
    "    tokenized_examples = tokenizer(examples[\"review\"], max_length=128, truncation=True)\n",
    "    tokenized_examples[\"labels\"] = examples[\"label\"]\n",
    "    return tokenized_examples\n",
    "\n",
    "tokenized_datasets = datasets.map(process_function, batched=True, remove_columns=datasets[\"train\"].column_names)\n",
    "tokenized_datasets"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step5 创建模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at /data/models/huggingface/rbt3 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = BertForSequenceClassification.from_pretrained(pretrain_model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertConfig {\n",
       "  \"_name_or_path\": \"/data/models/huggingface/rbt3\",\n",
       "  \"architectures\": [\n",
       "    \"BertForMaskedLM\"\n",
       "  ],\n",
       "  \"attention_probs_dropout_prob\": 0.1,\n",
       "  \"classifier_dropout\": null,\n",
       "  \"directionality\": \"bidi\",\n",
       "  \"hidden_act\": \"gelu\",\n",
       "  \"hidden_dropout_prob\": 0.1,\n",
       "  \"hidden_size\": 768,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 3072,\n",
       "  \"layer_norm_eps\": 1e-12,\n",
       "  \"max_position_embeddings\": 512,\n",
       "  \"model_type\": \"bert\",\n",
       "  \"num_attention_heads\": 12,\n",
       "  \"num_hidden_layers\": 3,\n",
       "  \"output_past\": true,\n",
       "  \"pad_token_id\": 0,\n",
       "  \"pooler_fc_size\": 768,\n",
       "  \"pooler_num_attention_heads\": 12,\n",
       "  \"pooler_num_fc_layers\": 3,\n",
       "  \"pooler_size_per_head\": 128,\n",
       "  \"pooler_type\": \"first_token_transform\",\n",
       "  \"position_embedding_type\": \"absolute\",\n",
       "  \"transformers_version\": \"4.43.3\",\n",
       "  \"type_vocab_size\": 2,\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 21128\n",
       "}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.config"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step6 创建评估函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "path ./metric_accuracy.py None DownloadMode.REUSE_DATASET_IF_EXISTS\n",
      "path ./metric_f1.py None DownloadMode.REUSE_DATASET_IF_EXISTS\n"
     ]
    }
   ],
   "source": [
    "import evaluate\n",
    "\n",
    "acc_metric = evaluate.load(\"./metric_accuracy.py\")\n",
    "f1_metirc = evaluate.load(\"./metric_f1.py\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_metric(eval_predict):\n",
    "    predictions, labels = eval_predict\n",
    "    predictions = predictions.argmax(axis=-1)\n",
    "    acc = acc_metric.compute(predictions=predictions, references=labels)\n",
    "    f1 = f1_metirc.compute(predictions=predictions, references=labels)\n",
    "    acc.update(f1)\n",
    "    return acc"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step7 创建TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\transformers\\training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "train_args = TrainingArguments(output_dir=save_dir,      # 输出文件夹\n",
    "                               per_device_train_batch_size=64,  # 训练时的batch_size\n",
    "                               per_device_eval_batch_size=128,  # 验证时的batch_size\n",
    "                               logging_steps=10,                # log 打印的频率\n",
    "                               evaluation_strategy=\"epoch\",     # 评估策略\n",
    "                               save_strategy=\"epoch\",           # 保存策略\n",
    "                               save_total_limit=3,              # 最大保存数\n",
    "                               learning_rate=2e-5,              # 学习率\n",
    "                               weight_decay=0.01,               # weight_decay\n",
    "                               metric_for_best_model=\"f1\",      # 设定评估指标\n",
    "                               load_best_model_at_end=True)     # 训练完成后加载最优模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'output_dir': '/data/logs/data_parallel_train',\n",
       " 'overwrite_output_dir': False,\n",
       " 'do_train': False,\n",
       " 'do_eval': True,\n",
       " 'do_predict': False,\n",
       " 'eval_strategy': <IntervalStrategy.EPOCH: 'epoch'>,\n",
       " 'prediction_loss_only': False,\n",
       " 'per_device_train_batch_size': 64,\n",
       " 'per_device_eval_batch_size': 128,\n",
       " 'per_gpu_train_batch_size': None,\n",
       " 'per_gpu_eval_batch_size': None,\n",
       " 'gradient_accumulation_steps': 1,\n",
       " 'eval_accumulation_steps': None,\n",
       " 'eval_delay': 0,\n",
       " 'torch_empty_cache_steps': None,\n",
       " 'learning_rate': 2e-05,\n",
       " 'weight_decay': 0.01,\n",
       " 'adam_beta1': 0.9,\n",
       " 'adam_beta2': 0.999,\n",
       " 'adam_epsilon': 1e-08,\n",
       " 'max_grad_norm': 1.0,\n",
       " 'num_train_epochs': 3.0,\n",
       " 'max_steps': -1,\n",
       " 'lr_scheduler_type': <SchedulerType.LINEAR: 'linear'>,\n",
       " 'lr_scheduler_kwargs': {},\n",
       " 'warmup_ratio': 0.0,\n",
       " 'warmup_steps': 0,\n",
       " 'log_level': 'passive',\n",
       " 'log_level_replica': 'warning',\n",
       " 'log_on_each_node': True,\n",
       " 'logging_dir': '/data/logs/data_parallel_train\\\\runs\\\\Oct16_00-23-09_WIN-20240216IDH',\n",
       " 'logging_strategy': <IntervalStrategy.STEPS: 'steps'>,\n",
       " 'logging_first_step': False,\n",
       " 'logging_steps': 10,\n",
       " 'logging_nan_inf_filter': True,\n",
       " 'save_strategy': <IntervalStrategy.EPOCH: 'epoch'>,\n",
       " 'save_steps': 500,\n",
       " 'save_total_limit': 3,\n",
       " 'save_safetensors': True,\n",
       " 'save_on_each_node': False,\n",
       " 'save_only_model': False,\n",
       " 'restore_callback_states_from_checkpoint': False,\n",
       " 'no_cuda': False,\n",
       " 'use_cpu': False,\n",
       " 'use_mps_device': False,\n",
       " 'seed': 42,\n",
       " 'data_seed': None,\n",
       " 'jit_mode_eval': False,\n",
       " 'use_ipex': False,\n",
       " 'bf16': False,\n",
       " 'fp16': False,\n",
       " 'fp16_opt_level': 'O1',\n",
       " 'half_precision_backend': 'auto',\n",
       " 'bf16_full_eval': False,\n",
       " 'fp16_full_eval': False,\n",
       " 'tf32': None,\n",
       " 'local_rank': 0,\n",
       " 'ddp_backend': None,\n",
       " 'tpu_num_cores': None,\n",
       " 'tpu_metrics_debug': False,\n",
       " 'debug': [],\n",
       " 'dataloader_drop_last': False,\n",
       " 'eval_steps': None,\n",
       " 'dataloader_num_workers': 0,\n",
       " 'dataloader_prefetch_factor': 2,\n",
       " 'past_index': -1,\n",
       " 'run_name': '/data/logs/data_parallel_train',\n",
       " 'disable_tqdm': False,\n",
       " 'remove_unused_columns': True,\n",
       " 'label_names': None,\n",
       " 'load_best_model_at_end': True,\n",
       " 'metric_for_best_model': 'f1',\n",
       " 'greater_is_better': True,\n",
       " 'ignore_data_skip': False,\n",
       " 'fsdp': [],\n",
       " 'fsdp_min_num_params': 0,\n",
       " 'fsdp_config': {'min_num_params': 0,\n",
       "  'xla': False,\n",
       "  'xla_fsdp_v2': False,\n",
       "  'xla_fsdp_grad_ckpt': False},\n",
       " 'fsdp_transformer_layer_cls_to_wrap': None,\n",
       " 'accelerator_config': AcceleratorConfig(split_batches=False, dispatch_batches=None, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False),\n",
       " 'deepspeed': None,\n",
       " 'label_smoothing_factor': 0.0,\n",
       " 'optim': <OptimizerNames.ADAMW_TORCH: 'adamw_torch'>,\n",
       " 'optim_args': None,\n",
       " 'adafactor': False,\n",
       " 'group_by_length': False,\n",
       " 'length_column_name': 'length',\n",
       " 'report_to': ['tensorboard'],\n",
       " 'ddp_find_unused_parameters': None,\n",
       " 'ddp_bucket_cap_mb': None,\n",
       " 'ddp_broadcast_buffers': None,\n",
       " 'dataloader_pin_memory': True,\n",
       " 'dataloader_persistent_workers': False,\n",
       " 'skip_memory_metrics': True,\n",
       " 'use_legacy_prediction_loop': False,\n",
       " 'push_to_hub': False,\n",
       " 'resume_from_checkpoint': None,\n",
       " 'hub_model_id': None,\n",
       " 'hub_strategy': <HubStrategy.EVERY_SAVE: 'every_save'>,\n",
       " 'hub_token': None,\n",
       " 'hub_private_repo': False,\n",
       " 'hub_always_push': False,\n",
       " 'gradient_checkpointing': False,\n",
       " 'gradient_checkpointing_kwargs': None,\n",
       " 'include_inputs_for_metrics': False,\n",
       " 'eval_do_concat_batches': True,\n",
       " 'fp16_backend': 'auto',\n",
       " 'evaluation_strategy': 'epoch',\n",
       " 'push_to_hub_model_id': None,\n",
       " 'push_to_hub_organization': None,\n",
       " 'push_to_hub_token': None,\n",
       " 'mp_parameters': '',\n",
       " 'auto_find_batch_size': False,\n",
       " 'full_determinism': False,\n",
       " 'torchdynamo': None,\n",
       " 'ray_scope': 'last',\n",
       " 'ddp_timeout': 1800,\n",
       " 'torch_compile': False,\n",
       " 'torch_compile_backend': None,\n",
       " 'torch_compile_mode': None,\n",
       " 'dispatch_batches': None,\n",
       " 'split_batches': None,\n",
       " 'include_tokens_per_second': False,\n",
       " 'include_num_input_tokens_seen': False,\n",
       " 'neftune_noise_alpha': None,\n",
       " 'optim_target_modules': None,\n",
       " 'batch_eval_metrics': False,\n",
       " 'eval_on_start': False,\n",
       " 'eval_use_gather_object': False,\n",
       " 'distributed_state': Distributed environment: NO\n",
       " Num processes: 1\n",
       " Process index: 0\n",
       " Local process index: 0\n",
       " Device: cuda,\n",
       " '_n_gpu': 2,\n",
       " '__cached__setup_devices': device(type='cuda', index=0),\n",
       " 'deepspeed_plugin': None}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hasattr(train_args, \"_n_gpu\")\n",
    "train_args.__dict__"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step8 创建Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorWithPadding\n",
    "trainer = Trainer(model=model, \n",
    "                  args=train_args, \n",
    "                  train_dataset=tokenized_datasets[\"train\"], \n",
    "                  eval_dataset=tokenized_datasets[\"test\"], \n",
    "                  data_collator=DataCollatorWithPadding(tokenizer=tokenizer),\n",
    "                  compute_metrics=eval_metric)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step9 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6077d46e45224a64bf5539aaf633b3a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/165 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\torch\\nn\\parallel\\_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\torch\\cuda\\nccl.py:15: UserWarning: PyTorch is not compiled with NCCL support\n",
      "  warnings.warn('PyTorch is not compiled with NCCL support')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.6468, 'grad_norm': 1.4253441095352173, 'learning_rate': 1.8787878787878792e-05, 'epoch': 0.18}\n",
      "{'loss': 0.5445, 'grad_norm': 2.7792410850524902, 'learning_rate': 1.7575757575757576e-05, 'epoch': 0.36}\n",
      "{'loss': 0.4366, 'grad_norm': 1.4885610342025757, 'learning_rate': 1.6363636363636366e-05, 'epoch': 0.55}\n",
      "{'loss': 0.3514, 'grad_norm': 1.2442349195480347, 'learning_rate': 1.5151515151515153e-05, 'epoch': 0.73}\n",
      "{'loss': 0.3776, 'grad_norm': 2.7849442958831787, 'learning_rate': 1.3939393939393942e-05, 'epoch': 0.91}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02ec6f60e3bb427092d94b3d4bc70f69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.32842373847961426, 'eval_accuracy': 0.861003861003861, 'eval_f1': 0.8988764044943821, 'eval_runtime': 0.6775, 'eval_samples_per_second': 1146.846, 'eval_steps_per_second': 5.904, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\torch\\nn\\parallel\\_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\torch\\cuda\\nccl.py:15: UserWarning: PyTorch is not compiled with NCCL support\n",
      "  warnings.warn('PyTorch is not compiled with NCCL support')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3281, 'grad_norm': 1.9412095546722412, 'learning_rate': 1.2727272727272728e-05, 'epoch': 1.09}\n",
      "{'loss': 0.2935, 'grad_norm': 3.138672113418579, 'learning_rate': 1.1515151515151517e-05, 'epoch': 1.27}\n",
      "{'loss': 0.2912, 'grad_norm': 3.6680994033813477, 'learning_rate': 1.0303030303030304e-05, 'epoch': 1.45}\n",
      "{'loss': 0.3038, 'grad_norm': 1.9130064249038696, 'learning_rate': 9.090909090909091e-06, 'epoch': 1.64}\n",
      "{'loss': 0.2925, 'grad_norm': 2.1191470623016357, 'learning_rate': 7.87878787878788e-06, 'epoch': 1.82}\n",
      "{'loss': 0.2885, 'grad_norm': 1.7933403253555298, 'learning_rate': 6.666666666666667e-06, 'epoch': 2.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dcd1d1493d0e4ff9b4114d028b63cb8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.2840496301651001, 'eval_accuracy': 0.8725868725868726, 'eval_f1': 0.9065155807365439, 'eval_runtime': 0.6428, 'eval_samples_per_second': 1208.852, 'eval_steps_per_second': 6.223, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\torch\\nn\\parallel\\_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\torch\\cuda\\nccl.py:15: UserWarning: PyTorch is not compiled with NCCL support\n",
      "  warnings.warn('PyTorch is not compiled with NCCL support')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2573, 'grad_norm': 1.6448779106140137, 'learning_rate': 5.4545454545454545e-06, 'epoch': 2.18}\n",
      "{'loss': 0.2695, 'grad_norm': 3.301877737045288, 'learning_rate': 4.242424242424243e-06, 'epoch': 2.36}\n",
      "{'loss': 0.2394, 'grad_norm': 1.931391954421997, 'learning_rate': 3.0303030303030305e-06, 'epoch': 2.55}\n",
      "{'loss': 0.2758, 'grad_norm': 1.6898844242095947, 'learning_rate': 1.8181818181818183e-06, 'epoch': 2.73}\n",
      "{'loss': 0.2793, 'grad_norm': 1.9433966875076294, 'learning_rate': 6.060606060606061e-07, 'epoch': 2.91}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\torch\\nn\\parallel\\_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9206de080b84d4f847b6d58b128b2de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.26441821455955505, 'eval_accuracy': 0.888030888030888, 'eval_f1': 0.9159420289855073, 'eval_runtime': 0.6439, 'eval_samples_per_second': 1206.786, 'eval_steps_per_second': 6.213, 'epoch': 3.0}\n",
      "{'train_runtime': 53.8716, 'train_samples_per_second': 389.148, 'train_steps_per_second': 3.063, 'train_loss': 0.33960448322874126, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=165, training_loss=0.33960448322874126, metrics={'train_runtime': 53.8716, 'train_samples_per_second': 389.148, 'train_steps_per_second': 3.063, 'total_flos': 351909933963264.0, 'train_loss': 0.33960448322874126, 'epoch': 3.0})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step10 模型评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\torch\\nn\\parallel\\_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd8dd733dcb74ccba1699e65138e24ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.26441821455955505,\n",
       " 'eval_accuracy': 0.888030888030888,\n",
       " 'eval_f1': 0.9159420289855073,\n",
       " 'eval_runtime': 0.6414,\n",
       " 'eval_samples_per_second': 1211.398,\n",
       " 'eval_steps_per_second': 6.236,\n",
       " 'epoch': 3.0}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate(tokenized_datasets[\"test\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformers",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
